################################################################################
## LOAD PACKAGES
################################################################################

library(shiny)
library(shinyBS)
library(shinyWidgets)
library(shinyjs)
library(dslabs)
library(tidyverse)
library(shinydashboard)
library(plotly)
library(gganimate)
library(ggthemes)
library(DT)
#library(anomalize)
library(timetk)
library(bsplus)
library(bslib)
library(readxl)
library(foreign)
library(ggtext)
library(ggsci)
library(haven)
library(lavaan)
library(lubridate)
library(cleaner)
library(rlang)
library(shinythemes)
library(forecast)
library(stringi)
library(stringr)
library(fontawesome)
library(htmltools)
library(scales)
library(sqldf)
library(shinyalert)
library(psych)
library(magick)
library(mailtoR)
library(later)
library(bsicons)
library(ShinyItemAnalysis)
library(sf)
library(viridis)
library(Hmisc)
library(leaflet)
library(ggimage)
library(grid)
library(gridExtra)
library(png)
library(base64enc)
library(readxl)
library(xlsx)
library(forcats)
library(pdftools)
library(janitor)
library(stringr)
library(LaF)
library(shinymanager)
library(data.table)
library(EFAtools)
library(EFA.dimensions)
library(broom)
library(tibble)
library(xtable)
library(corrplot)

################################################################################
## SET WORKING DIRECTORY
################################################################################

getwd()

#setwd() # Set working directory here

################################################################################
## LOAD STEMTEST DATA
################################################################################

# Load full dataset

stemtest.full.df <- fread("responses-volwassenen-10-06-2024.csv")

stemtest.full.df <- stemtest.full.df %>%
  mutate_if(is.logical, as.character)

# Remove timestamp from date

stemtest.full.df$createdAt <- as.POSIXct(stemtest.full.df$createdAt,format='%Y-%m-%d',tz="UTC")

# Drop duration columns

stemtest.full.df <- stemtest.full.df %>% select(-contains("_duration"))

# Rename columns

stemtest.full.df <- rename(stemtest.full.df,
                           c('language'='locale',
                             'vaa'='level',
                             'date'='createdAt'))

# Recode language

stemtest.full.df$language[stemtest.full.df$language=="fr"] <- "French"
stemtest.full.df$language[stemtest.full.df$language=="nl"] <- "Dutch"

# Recode VAA

stemtest.full.df$vaa[stemtest.full.df$vaa=="base"] <- "General"
stemtest.full.df$vaa[stemtest.full.df$vaa=="federal"] <- "Federal"
stemtest.full.df$vaa[stemtest.full.df$vaa=="brussels"] <- "Brussels"
stemtest.full.df$vaa[stemtest.full.df$vaa=="european"] <- "EU"
stemtest.full.df$vaa[stemtest.full.df$vaa=="regional" & stemtest.full.df$language=="French"] <- "Wallonia"
stemtest.full.df$vaa[stemtest.full.df$vaa=="regional" & stemtest.full.df$language=="Dutch"] <- "Flanders"

stemtest.full.df$date <- as.Date(stemtest.full.df$date)

stemtest.general.nl.df <- stemtest.full.df %>% filter(vaa == "General" & language == "Dutch")
stemtest.general.nl.df <- stemtest.general.nl.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.general.nl.df <- stemtest.general.nl.df %>% mutate_if(is.logical, as.character)

stemtest.general.fr.df <- stemtest.full.df %>% filter(vaa == "General" & language == "French")
stemtest.general.fr.df <- stemtest.general.fr.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.general.fr.df <- stemtest.general.fr.df %>% mutate_if(is.logical, as.character)

stemtest.federal.nl.df <- stemtest.full.df %>% filter(vaa == "Federal" & language == "Dutch")
stemtest.federal.nl.df <- stemtest.federal.nl.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.federal.nl.df <- stemtest.federal.nl.df %>% mutate_if(is.logical, as.character)

stemtest.federal.fr.df <- stemtest.full.df %>% filter(vaa == "Federal" & language == "French")
stemtest.federal.fr.df <- stemtest.federal.fr.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.federal.frdf <- stemtest.federal.fr.df %>% mutate_if(is.logical, as.character)

stemtest.brussels.nl.df <- stemtest.full.df %>% filter(vaa == "Brussels" & language == "Dutch")
stemtest.brussels.nl.df <- stemtest.brussels.nl.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.brussels.nl.df <- stemtest.brussels.nl.df %>% mutate_if(is.logical, as.character)

stemtest.brussels.fr.df <- stemtest.full.df %>% filter(vaa == "Brussels" & language == "French")
stemtest.brussels.fr.df <- stemtest.brussels.fr.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.brussels.fr.df <- stemtest.brussels.fr.df %>% mutate_if(is.logical, as.character)

stemtest.eu.nl.df <- stemtest.full.df %>% filter(vaa == "EU" & language == "Dutch")
stemtest.eu.nl.df <- stemtest.eu.nl.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.eu.nl.df <- stemtest.eu.nl.df %>% mutate_if(is.logical, as.character)

stemtest.eu.fr.df <- stemtest.full.df %>% filter(vaa == "EU" & language == "French")
stemtest.eu.fr.df <- stemtest.eu.fr.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.eu.fr.df <- stemtest.eu.fr.df %>% mutate_if(is.logical, as.character)

stemtest.wallonia.fr.df <- stemtest.full.df %>% filter(vaa == "Wallonia")
stemtest.wallonia.fr.df <- stemtest.wallonia.fr.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.wallonia.fr.df <- stemtest.wallonia.fr.df %>% mutate_if(is.logical, as.character)

stemtest.flanders.nl.df <- stemtest.full.df %>% filter(vaa == "Flanders")
stemtest.flanders.nl.df <- stemtest.flanders.nl.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
stemtest.flanders.nl.df <- stemtest.flanders.nl.df %>% mutate_if(is.logical, as.character)

write.csv(stemtest.general.nl.df, 'stemtest_general_nl_df.csv', row.names = FALSE)
write.csv(stemtest.general.fr.df, 'stemtest_general_fr_df.csv', row.names = FALSE)

write.csv(stemtest.federal.nl.df, 'stemtest_federal_nl_df.csv', row.names = FALSE)
write.csv(stemtest.federal.fr.df, 'stemtest_federal_fr_df.csv', row.names = FALSE)

write.csv(stemtest.brussels.nl.df, 'stemtest_brussels_nl_df.csv', row.names = FALSE)
write.csv(stemtest.brussels.fr.df, 'stemtest_brussels_fr_df.csv', row.names = FALSE)

write.csv(stemtest.eu.nl.df, 'stemtest_eu_nl_df.csv', row.names = FALSE)
write.csv(stemtest.eu.fr.df, 'stemtest_eu_fr_df.csv', row.names = FALSE)

write.csv(stemtest.wallonia.fr.df, 'stemtest_wallonia_fr_df.csv', row.names = FALSE)
write.csv(stemtest.flanders.nl.df, 'stemtest_flanders_nl_df.csv', row.names = FALSE)

################################################################################
## LOAD YOUTH DATA
################################################################################

# Load full dataset

youth.full.df <- fread("responses-jongeren-10-06-2024.csv")

youth.full.df <- youth.full.df %>%
  mutate_if(is.logical, as.character)

# Remove timestamp from date

youth.full.df$createdAt <- as.POSIXct(youth.full.df$createdAt,format='%Y-%m-%d',tz="UTC")

# Drop duration columns

youth.full.df <- youth.full.df %>% select(-contains("_duration"))

# Rename columns

youth.full.df <- rename(youth.full.df,
                        c('language'='locale',
                          'vaa'='level',
                          'date'='createdAt'))

# Recode language

youth.full.df$language[youth.full.df$language=="fr"] <- "French"
youth.full.df$language[youth.full.df$language=="nl"] <- "Dutch"

# Recode VAA

youth.full.df$vaa[youth.full.df$language=="French"] <- "Youth FR"
youth.full.df$vaa[youth.full.df$language=="Dutch"] <- "Youth NL"

youth.full.df$date <- as.Date(youth.full.df$date)

youth.nl.df <- youth.full.df %>% filter(language == "Dutch")
youth.nl.df <- youth.nl.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
youth.nl.df <- youth.nl.df %>% mutate_if(is.logical, as.character)

youth.fr.df <- youth.full.df %>% filter(language == "French")
youth.fr.df <- youth.fr.df %>% select(where(~ !is.character(.) | !all(is.na(.) | . == "")))
youth.fr.df <- youth.fr.df %>% mutate_if(is.logical, as.character)

write.csv(youth.nl.df, 'stemtest_youth_nl_df.csv', row.names = FALSE)
write.csv(youth.fr.df, 'stemtest_youth_fr_df.csv', row.names = FALSE)